﻿Imports System.Net
Imports System.Text
Imports System.Web
Imports System.Runtime.Serialization
Imports System.Xml
Imports System.IO
Imports Sgml
Imports System.Text.RegularExpressions
Imports System.Runtime.Serialization.Json

Namespace Scraping
    Partial Class TwitterClient
        Implements ITwitterClient
        ''' <summary>
        ''' タイムラインの種類を表現します。
        ''' </summary>
        ''' <remarks></remarks>
        Private Enum TimelineType
            [Public]
            Home
            Replies
            Inbox
            Sent
            User
            Favorites
        End Enum

        ''' <summary>
        ''' タイムラインを取得した結果として返されるJSONを格納します。
        ''' </summary>
        ''' <remarks></remarks>
        <DataContract()> _
        Private Class GetTimeLineJsonResult
            Private _pagination As String
            <DataMember(Name:="#pagination")> _
            Public Property Pagination() As String
                Get
                    Return _pagination
                End Get
                Set(ByVal value As String)
                    _pagination = value
                End Set
            End Property

            Private _timeline As String
            <DataMember(Name:="#timeline")> _
            Public Property Timeline() As String
                Get
                    Return _timeline
                End Get
                Set(ByVal value As String)
                    _timeline = value
                End Set
            End Property
        End Class

        Private _timelinePath As New Dictionary(Of TimelineType, String)

        Private Sub InitializeTimeline(ByVal sender As Object, ByVal e As EventArgs) Handles Me.OnInitialize
            _timelinePath.Add(TimelineType.Public, "/public_timeline")
            _timelinePath.Add(TimelineType.Home, "/home")
            _timelinePath.Add(TimelineType.Replies, "/replies")
            _timelinePath.Add(TimelineType.Inbox, "/inbox")
            _timelinePath.Add(TimelineType.Sent, "/sent")
            _timelinePath.Add(TimelineType.User, "/{0}")
            _timelinePath.Add(TimelineType.Favorites, "/{0}/favorites")
        End Sub

        ''' <summary>
        ''' パブリック(Public)タイムラインを取得します。
        ''' </summary>
        ''' <param name="maxPage">取得する最大ページ数</param>
        ''' <param name="lastStatus">取得済みの最新のステータス</param>
        ''' <returns>TwitterStatusのコレクション</returns>
        ''' <remarks></remarks>
        Public Function GetPublicTimeline(ByVal maxPage As Integer, ByVal lastStatus As TwitterStatus) As IList(Of TwitterStatus) Implements ITwitterClient.GetPublicTimeline
            Return GetPublicTimeline(maxPage, lastStatus.StatusID)
        End Function

        ''' <summary>
        ''' パブリック(Public)タイムラインを取得します。
        ''' </summary>
        ''' <param name="maxPage">取得する最大ページ数</param>
        ''' <param name="lastStatusID">取得済みの最新のステータスの固有ID</param>
        ''' <returns>TwitterStatusのコレクション</returns>
        ''' <remarks></remarks>
        Public Function GetPublicTimeline(ByVal maxPage As Integer, ByVal lastStatusID As Long) As IList(Of TwitterStatus) Implements ITwitterClient.GetPublicTimeline
            Return GetTimeline(maxPage, lastStatusID, TimelineType.Public, Nothing)
        End Function

        ''' <summary>
        ''' ホーム(Recent/Friends)タイムラインを取得します。
        ''' </summary>
        ''' <param name="maxPage">取得する最大ページ数</param>
        ''' <param name="lastStatus">取得済みの最新のステータス</param>
        ''' <returns>TwitterStatusのコレクション</returns>
        ''' <remarks></remarks>
        Public Function GetHomeTimeline(ByVal maxPage As Integer, ByVal lastStatus As TwitterStatus) As IList(Of TwitterStatus) Implements ITwitterClient.GetHomeTimeline
            Return GetHomeTimeline(maxPage, lastStatus.StatusID)
        End Function

        ''' <summary>
        ''' ホーム(Recent/Friends)タイムラインを取得します。
        ''' </summary>
        ''' <param name="maxPage">取得する最大ページ数</param>
        ''' <param name="lastStatusID">取得済みの最新のステータスの固有ID</param>
        ''' <returns>TwitterStatusのコレクション</returns>
        ''' <remarks></remarks>
        Public Function GetHomeTimeline(ByVal maxPage As Integer, ByVal lastStatusID As Long) As IList(Of TwitterStatus) Implements ITwitterClient.GetHomeTimeline
            Return GetTimeline(maxPage, lastStatusID, TimelineType.Home, Nothing)
        End Function

        ''' <summary>
        ''' 返信(Replies/Memento)タイムラインを取得します。
        ''' </summary>
        ''' <param name="maxPage">取得する最大ページ数</param>
        ''' <param name="lastStatus">取得済みの最新のステータス</param>
        ''' <returns>TwitterStatusのコレクション</returns>
        ''' <remarks></remarks>
        Public Function GetRepliesTimeline(ByVal maxPage As Integer, ByVal lastStatus As TwitterStatus) As IList(Of TwitterStatus) Implements ITwitterClient.GetRepliesTimeline
            Return GetRepliesTimeline(maxPage, lastStatus.StatusID)
        End Function

        ''' <summary>
        ''' 返信(Replies/Memento)タイムラインを取得します。
        ''' </summary>
        ''' <param name="maxPage">取得する最大ページ数</param>
        ''' <param name="lastStatusID">取得済みの最新のステータスの固有ID</param>
        ''' <returns>TwitterStatusのコレクション</returns>
        ''' <remarks></remarks>
        Public Function GetRepliesTimeline(ByVal maxPage As Integer, ByVal lastStatusID As Long) As IList(Of TwitterStatus) Implements ITwitterClient.GetRepliesTimeline
            Return GetTimeline(maxPage, lastStatusID, TimelineType.Replies, Nothing)
        End Function

        ''' <summary>
        ''' 受信ダイレクトメッセージ(Inbox)タイムラインを取得します。
        ''' </summary>
        ''' <param name="maxPage">取得する最大ページ数</param>
        ''' <param name="lastStatus">取得済みの最新のステータス</param>
        ''' <returns>TwitterStatusのコレクション</returns>
        ''' <remarks></remarks>
        Public Function GetInboxTimeline(ByVal maxPage As Integer, ByVal lastStatus As TwitterStatus) As IList(Of TwitterStatus) Implements ITwitterClient.GetInboxTimeline
            Return GetInboxTimeline(maxPage, lastStatus.StatusID)
        End Function

        ''' <summary>
        '''  受信ダイレクトメッセージ(Inbox)タイムラインを取得します。
        ''' </summary>
        ''' <param name="maxPage">取得する最大ページ数</param>
        ''' <param name="lastStatusID">取得済みの最新のステータスの固有ID</param>
        ''' <returns>TwitterStatusのコレクション</returns>
        ''' <remarks></remarks>
        Public Function GetInboxTimeline(ByVal maxPage As Integer, ByVal lastStatusID As Long) As IList(Of TwitterStatus) Implements ITwitterClient.GetInboxTimeline
            Return GetTimeline(maxPage, lastStatusID, TimelineType.Inbox, Nothing)
        End Function

        ''' <summary>
        ''' 送信ダイレクトメッセージ(Sent)タイムラインを取得します。
        ''' </summary>
        ''' <param name="maxPage">取得する最大ページ数</param>
        ''' <param name="lastStatus">取得済みの最新のステータス</param>
        ''' <returns>TwitterStatusのコレクション</returns>
        ''' <remarks></remarks>
        Public Function GetSentTimeline(ByVal maxPage As Integer, ByVal lastStatus As TwitterStatus) As IList(Of TwitterStatus) Implements ITwitterClient.GetSentTimeline
            Return GetSentTimeline(maxPage, lastStatus.StatusID)
        End Function

        ''' <summary>
        ''' 送信ダイレクトメッセージ(Sent)タイムラインを取得します。
        ''' </summary>
        ''' <param name="maxPage">取得する最大ページ数</param>
        ''' <param name="lastStatusID">取得済みの最新のステータスの固有ID</param>
        ''' <returns>TwitterStatusのコレクション</returns>
        ''' <remarks></remarks>
        Public Function GetSentTimeline(ByVal maxPage As Integer, ByVal lastStatusID As Long) As IList(Of TwitterStatus) Implements ITwitterClient.GetSentTimeline
            Return GetTimeline(maxPage, lastStatusID, TimelineType.Sent, Nothing)
        End Function

        ''' <summary>
        ''' ユーザーのタイムラインを取得します。
        ''' </summary>
        ''' <param name="maxPage">取得する最大ページ数</param>
        ''' <param name="lastStatus">取得済みの最新のステータス</param>
        ''' <returns>TwitterStatusのコレクション</returns>
        ''' <remarks></remarks>
        Public Function GetUserTimeline(ByVal maxPage As Integer, ByVal lastStatus As TwitterStatus, ByVal user As TwitterUser) As IList(Of TwitterStatus) Implements ITwitterClient.GetUserTimeline
            Return GetUserTimeline(maxPage, lastStatus.StatusID, user.UserName)
        End Function

        ''' <summary>
        ''' ユーザーのタイムラインを取得します。
        ''' </summary>
        ''' <param name="maxPage">取得する最大ページ数</param>
        ''' <param name="lastStatusID">取得済みの最新のステータスの固有ID</param>
        ''' <returns>TwitterStatusのコレクション</returns>
        ''' <remarks></remarks>
        Public Function GetUserTimeline(ByVal maxPage As Integer, ByVal lastStatusID As Long, ByVal userName As String) As IList(Of TwitterStatus) Implements ITwitterClient.GetUserTimeline
            Return GetTimeline(maxPage, lastStatusID, TimelineType.User, userName)
        End Function

        ''' <summary>
        ''' ユーザーのお気に入りを取得します。
        ''' </summary>
        ''' <param name="maxPage">取得する最大ページ数</param>
        ''' <param name="lastStatus">取得済みの最新のステータス</param>
        ''' <returns>TwitterStatusのコレクション</returns>
        ''' <remarks></remarks>
        Public Function GetUserFavorites(ByVal maxPage As Integer, ByVal lastStatus As TwitterStatus, ByVal user As TwitterUser) As IList(Of TwitterStatus) Implements ITwitterClient.GetUserFavorites
            Return GetUserFavorites(maxPage, lastStatus.StatusID, user.UserName)
        End Function

        ''' <summary>
        ''' ユーザーのお気に入りを取得します。
        ''' </summary>
        ''' <param name="maxPage">取得する最大ページ数</param>
        ''' <param name="lastStatusID">取得済みの最新のステータスの固有ID</param>
        ''' <returns>TwitterStatusのコレクション</returns>
        ''' <remarks></remarks>
        Public Function GetUserFavorites(ByVal maxPage As Integer, ByVal lastStatusID As Long, ByVal userName As String) As IList(Of TwitterStatus) Implements ITwitterClient.GetUserFavorites
            Return GetTimeline(maxPage, lastStatusID, TimelineType.Favorites, userName)
        End Function

        Private Function GetTimeline(ByVal maxPage As Integer, ByVal lastStatusID As Long, ByVal type As TimelineType) As IList(Of TwitterStatus)
            Return GetTimeline(maxPage, lastStatusID, type, Nothing)
        End Function

        Private Shared _createdAtRegex As New Regex("(?<=""created_at""\:"")(Sun|Mon|Tue|Wed|Thu|Fri|Sat) (?<mon>Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) (?<day>\d{2}) (?<hour>\d{2})\:(?<min>\d{2})\:(?<sec>\d{2}) \+\d{4} (?<year>\d{4})")
        Private Shared _idNullRegex As New Regex("(?<=id""\:)null")
        Private Shared _followingNullRegex As New Regex("(?<=following""\:)null")
        Private Function GetTimeline(ByVal maxPage As Integer, ByVal lastStatusID As Long, ByVal type As TimelineType, ByVal userName As String) As IList(Of TwitterStatus)
            If Not IsLogined Then
                Throw New InvalidOperationException(StrThisInstanceIsNotLoginedYet)
            End If
            If (type = TimelineType.User OrElse type = TimelineType.Favorites) AndAlso userName Is Nothing Then
                Throw New ArgumentNullException("userName", "typeがTimelineType.UserまたはTimelineType.UserFavoritesの場合userNameにNullを指定することはできません。")
            End If

            Dim lstStatus As New List(Of TwitterStatus)
            Dim maxId As Long = 0
            If maxPage = 0 Then
                maxPage = 1
            End If
            Dim pageNo As Integer = 1
            Dim user As TwitterUser = Nothing
            If (type = TimelineType.User) Then
                user = GetUserForTimeline(userName)
                If user Is Nothing Then
                    Return Nothing
                End If
            End If
            While pageNo <= maxPage
                Dim url As String
                url = String.Format("{0}{1}?twttr=true&page={2}", My.Resources.TwitterUrl, String.Format(_timelinePath(type), userName), pageNo)
                If (maxId > 0) Then
                    url += String.Format("&max_id={0}", maxId)
                End If
                Dim urlReffer As String = String.Format("{0}{1}", My.Resources.TwitterUrl, _timelinePath(type))
                If pageNo = 1 Then
                    Dim reqReffer As HttpWebRequest = DirectCast(WebRequest.Create(url), HttpWebRequest)
                    reqReffer.SetTwitterGetHeaders(My.Resources.TwitterUrl, _cookies)
                    Dim resReffer As HttpWebResponse = DirectCast(reqReffer.GetResponse(), HttpWebResponse)
                    resReffer.Close()
                End If
                Dim req As HttpWebRequest = DirectCast(WebRequest.Create(url), HttpWebRequest)
                req.SetTwitterXHRGetHeaders(urlReffer, _cookies)
                Dim jsonResult As GetTimeLineJsonResult
                Using res As HttpWebResponse = DirectCast(req.GetResponse(), HttpWebResponse)
                    Select Case res.StatusCode
                        Case HttpStatusCode.OK
                            Select Case type
                                Case TimelineType.Favorites
                                    If res.ContentLength > 2 Then
                                        Dim strRes As String
                                        Using srRes As StreamReader = New StreamReader(res.GetResponseStream())
                                            strRes = srRes.ReadToEnd()
                                        End Using
                                        strRes = _createdAtRegex.Replace(strRes, New MatchEvaluator(AddressOf DateMatchEvaluator))
                                        strRes = _idNullRegex.Replace(strRes, "0")
                                        strRes = _followingNullRegex.Replace(strRes, "false")
                                        Using sRes As Stream = New MemoryStream(Encoding.UTF8.GetBytes(strRes))
                                            Dim json As New DataContractJsonSerializer(GetType(TwitterStatus()))
                                            Dim tl As TwitterStatus() = DirectCast(json.ReadObject(sRes), TwitterStatus())
                                            For Each status As TwitterStatus In tl
                                                lstStatus.Add(status)
                                            Next
                                        End Using
                                    Else
                                        Exit While
                                    End If
                                Case Else
                                    jsonResult = res.GetResponseAsJson(Of GetTimeLineJsonResult)()
                            End Select
                        Case Else
                            Return Nothing
                    End Select
                End Using
                Dim xmlRes As XmlDocument = Nothing
                If type <> TimelineType.Favorites Then
                    If String.IsNullOrEmpty(jsonResult.Timeline) Then
                        Return Nothing
                    End If
                    xmlRes.LoadSgml(jsonResult.Timeline)
                    Dim statusListItems As XmlNodeList = xmlRes.SelectNodes("ol[@id='timeline']/li")
                    For Each node As XmlNode In statusListItems
                        Dim status As TwitterStatus
                        If (type <> TimelineType.User) Then
                            status = ParseStatusListItem(node, type, Nothing)
                        Else
                            status = ParseStatusListItem(node, type, user)
                        End If
                        If (status.StatusID = lastStatusID) Then
                            Exit While
                        End If
                        lstStatus.Add(status)
                    Next
                    If String.IsNullOrEmpty(jsonResult.Pagination) Then
                        Exit While
                    End If
                    xmlRes.LoadSgml(jsonResult.Pagination)
                    Dim pnAnchorNode As XmlNode = xmlRes.SelectSingleNode("/div[@id='pagination']/a")
                    If pnAnchorNode Is Nothing Then
                        Exit While
                    End If
                    Dim pnHref As String = HttpUtility.HtmlDecode(pnAnchorNode.Attributes("href").Value)
                    If (maxId = 0) Then
                        Dim bgnIdx As Integer = pnHref.IndexOf("?max_id=") + 8
                        maxId = CLng(pnHref.Substring(bgnIdx, pnHref.IndexOf("&", bgnIdx) - bgnIdx))
                    End If
                End If
                pageNo += 1
            End While
            Return lstStatus
        End Function

        Private Shared _dtUnixBegin As New DateTime(1970, 1, 1)
        Private Function DateMatchEvaluator(ByVal m As Match) As String
            Dim year As Integer = Integer.Parse(m.Groups("year").Value)
            Dim mon As Integer
            Select Case m.Groups("mon").Value
                Case "Jan"
                    mon = 1
                Case "Feb"
                    mon = 2
                Case "Mar"
                    mon = 3
                Case "Apr"
                    mon = 4
                Case "May"
                    mon = 5
                Case "Jun"
                    mon = 6
                Case "Jul"
                    mon = 7
                Case "Aug"
                    mon = 8
                Case "Sep"
                    mon = 9
                Case "Oct"
                    mon = 10
                Case "Nov"
                    mon = 11
                Case "Dec"
                    mon = 12
            End Select
            Dim day As Integer = Integer.Parse(m.Groups("day").Value)
            Dim hour As Integer = Integer.Parse(m.Groups("hour").Value)
            Dim min As Integer = Integer.Parse(m.Groups("min").Value)
            Dim sec As Integer = Integer.Parse(m.Groups("sec").Value)
            Dim dt = New DateTime(year, mon, day, hour, min, sec)
            Dim ret = String.Format("\/Date({0}+0900)\/", dt.Subtract(_dtUnixBegin).TotalMilliseconds)
            Return ret
        End Function

        ''' <summary>
        ''' レスポンスに含まれる投稿されたステータスを解析します。
        ''' </summary>
        ''' <param name="node">ステータスのルートノード</param>
        ''' <param name="type">タイムラインの種類</param>
        ''' <returns>解析されたステータス</returns>
        ''' <remarks></remarks>
        Private Shared Function ParseStatusListItem(ByVal node As XmlNode, ByVal type As TimelineType, ByVal user As TwitterUser) As TwitterStatus
            Dim status As TwitterStatus
            status = New TwitterStatus()
            status.CreatedAt = DateTime.Now 'TODO:Web経由では曖昧な値しか取れないため
            If type = TimelineType.Inbox Or type = TimelineType.Sent Then
                status.StatusID = CLng(node.Attributes("id").Value.Substring(15))
            Else
                status.StatusID = CLng(node.Attributes("id").Value.Substring(7))
            End If
            If (type <> TimelineType.User) Then
                Dim authorAnchorNode As XmlNode = node.SelectSingleNode("*//a[@class='url']")
                status.User.UserName = authorAnchorNode.Attributes("href").Value.Substring(19)
                Dim authorImageNode As XmlNode = authorAnchorNode.SelectSingleNode("img")
                status.User.DisplayName = authorImageNode.Attributes("alt").Value
                status.User.IconUrl = authorImageNode.Attributes("src").Value
                Dim bgnIndex As Integer = status.User.IconUrl.IndexOf("profile_images/")
                If (bgnIndex >= 0) Then
                    bgnIndex += 15
                    Dim len As Integer = status.User.IconUrl.IndexOf("/"c, bgnIndex) - bgnIndex
                    'TODO:画像が設定されていない場合のUserID
                    status.User.UserID = CLng(status.User.IconUrl.Substring(bgnIndex, len))
                End If
            Else
                status.User = user
            End If
            Dim statusBodyNode As XmlNode = node.SelectSingleNode("span[@class='status-body']")
            status.Text = statusBodyNode.SelectSingleNode("span[@class='entry-content']").InnerText
            Dim lockImageNode As XmlNode = statusBodyNode.SelectSingleNode("img[@class='lock']")
            If lockImageNode IsNot Nothing Then
                status.User.IsProtected = True
            End If
            Dim entryMetaNode As XmlNode = statusBodyNode.SelectSingleNode("span[@class='meta entry-meta']")
            If type <> TimelineType.Inbox And type <> TimelineType.Sent Then
                Dim fromNode = entryMetaNode.SelectSingleNode("span[contains(text(),'で')]")
                status.Source = fromNode.InnerText.TrimEnd("で"c)
                Dim replyToAnchorNode = entryMetaNode.SelectSingleNode("a[contains(text(),'宛')]")
                If replyToAnchorNode IsNot Nothing Then
                    status.ReplyUserName = replyToAnchorNode.InnerText.TrimEnd("宛"c)
                    status.ReplyStatusID = CLng(replyToAnchorNode.Attributes("href").Value.Substring(27 + status.ReplyUserName.Length))
                End If
                Dim favActionAnchorNode = node.SelectSingleNode("span[@class='actions']/div/a[contains(@class,'fav-action')]")
                status.IsFavorated = favActionAnchorNode.Attributes("class").Value.IndexOf("non") < 0
            End If
            Return status
        End Function

        Private Function GetUserForTimeline(ByVal userName As String) As TwitterUser
            Dim user As TwitterUser = New TwitterUser()
            Dim url As String = String.Format("{0}/{1}", My.Resources.TwitterUrl, userName)
            Dim req As HttpWebRequest = DirectCast(WebRequest.Create(url), HttpWebRequest)
            req.SetTwitterGetHeaders(Nothing, _cookies)
            Using res As HttpWebResponse = DirectCast(req.GetResponse(), HttpWebResponse)
                Select Case res.StatusCode
                    Case HttpStatusCode.OK
                        res.LoadCookies(_cookies)
                        Dim xmlRes As XmlDocument = res.GetResponseAsXHtml()
                        Dim xnm As XmlNamespaceManager = xmlRes.GetXHtmlNameSpaceManager("x")
                        Dim followActionsNode As XmlNode = xmlRes.SelectSingleNode("//x:div[starts-with(@id,'follow_actions_')]", xnm)
                        user.UserID = CLng(followActionsNode.Attributes("id").Value.Substring(15))
                        user.UserName = followActionsNode.Attributes("class").Value.Substring(15)
                        Dim iconImageNode As XmlNode = xmlRes.SelectSingleNode("//x:h2[@class='thumb clearfix']/x:a[starts-with(@href,'/account/profile_image/')]/x:img[@id='profile-image']", xnm)
                        user.IconUrl = iconImageNode.Attributes("src").Value
                        Dim displayNameNode As XmlNode = xmlRes.SelectSingleNode("//x:div[@id='profile']//x:ul/x:li/x:span[@class='fn']", xnm)
                        user.DisplayName = displayNameNode.InnerText.Trim
                    Case Else
                        Return Nothing
                End Select
            End Using
            Return user
        End Function
    End Class
End Namespace